﻿using System;
using System.Diagnostics;
using System.Threading;
using System.ServiceProcess;
using System.Collections.Generic;
using System.Text;
using System.Linq;
using HenIT.Data.SqlClient;

namespace EventScavenger
{
    public partial class EventScavenger : ServiceBase
    {
        public EventScavenger()
        {
            InitializeComponent();
        }

        #region Constants
        private const int INFINITERETRIES = -1; 
        #endregion

        #region Privates
        private static string ServiceEventSource = "Event scavenger";
        private static string perfCategoryThreads = "Event scavenger Threads";
        private static bool running = true;
        private static bool paused = false;
        private static bool recycling = false;
        private static int waitingTimeDecrement = 2000;
        private static Collector thisCollector = null;
        private static EventScavengerDAL eventLogDbDAL;
        private static List<Thread> pollingThreads;
        private static Mutex reportThreadToMaster = new Mutex();

        private static PerformanceCounter machineLogThreadCount = null;
        private static PerformanceCounter machineLogActiveThreadCount = null;
        private static PerformanceCounter eventEntriesPerSecond = null;
        private static PerformanceCounter errorsPerSecond = null;
        private static PerformanceCounter dbAccessPerSecond = null;
        private static PerformanceCounter dbInsertsPerSecond = null;
        private static PerformanceCounter dbDuplicateInsertsPerSecond = null;
        private static PerformanceCounter dbAccessDurationMilliseconds = null;

        private static string executionDurationTimeCounterName = "Execution Duration time";
        private static string isActiveIndicatorCounterName = "Thread activity indicator";
        #endregion

        #region Start/Stop/Pause/Continue of service
        protected override void OnStart(string[] args)
        {
#if DEBUG
            //The following code is simply to ease attaching the debugger to the service to debug the startup routine
            DateTime startTime = DateTime.Now;
            while ((!Debugger.IsAttached) && ((TimeSpan)DateTime.Now.Subtract(startTime)).TotalSeconds < 20 )  // Waiting until debugger is attached
            {
                RequestAdditionalTime(1000);  // Prevents the service from timeout
                Thread.Sleep(1000);           // Gives you time to attach the debugger   
            }
            // increase as needed to prevent timeouts
            RequestAdditionalTime(5000);     // for Debugging the OnStart method,     
#endif

            EventLog.WriteEntry(ServiceEventSource, "Starting Event Scavenger version:" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version
                + "\r\nRunning as Collector: " + Properties.Settings.Default.ColletorName
                , EventLogEntryType.Information, (int)EventIDCodes.SystemEvents);
            InitPerformanceCounters();

            new Thread(delegate()
                {
                    MasterThread();
                }
            ).Start();
        }

        protected override void OnStop()
        {
            running = false;
        }

        protected override void OnPause()
        {
            paused = true;
            base.OnPause();
        }

        protected override void OnContinue()
        {
            paused = false;
            base.OnContinue();
        }

        /// <summary>
        /// When the startup of the service fails it must be stopped asynchronisly
        /// </summary>
        /// <param name="reason">Text description of reason why service is shutting down</param>
        private static void HaltService(string reason)
        {
            EventLog.WriteEntry(ServiceEventSource, reason, EventLogEntryType.Error, (int)EventIDCodes.GeneralError);
            new Thread(delegate()
            {
                Thread.Sleep(5000);
                ServiceController srvController = new ServiceController("Event scavenger");
                srvController.Stop();
            }
           ).Start();
        }
        #endregion

        #region Master thread
        /// <summary>
        /// Main thread of service that spawns other threads and control recycling when enabled
        /// </summary>
        private static void MasterThread()
        {
            List<MachineLogEntry> machineLogs;
            int threadNo;
            bool isRecyclingEnabled = false;
            int recyclingCheckFrequency = 5;
            PerformanceCounter executeDurationCount = null;
            PerformanceCounter isActiveIndicator = null;
            Stopwatch sw = new Stopwatch();
            try
            {
                try
                {
                    executeDurationCount = InitializePerfCounterInstance(perfCategoryThreads, executionDurationTimeCounterName, "Master thread");
                    isActiveIndicator = InitializePerfCounterInstance(perfCategoryThreads, isActiveIndicatorCounterName, "Master thread");
                }
                catch (Exception ex)
                {
                    EventLog.WriteEntry(ServiceEventSource, "There was an error initializing the '" + perfCategoryThreads  + "' performance counters for the master thread.\r\n" + ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.GeneralError);
                }

                SetThreadCounterActiveState(isActiveIndicator, true);
                sw.Start(); //measuring set up time

                #region Database connectivity
                //Setting up data access object
                eventLogDbDAL = new EventScavengerDAL();
                eventLogDbDAL.CommandTimeout = Properties.Settings.Default.SqlCommandTimeOut;
                #region eventLogDbDAL events
                eventLogDbDAL.RaiseDBAccess += new RaiseDBAccessDelegate(eventLogDbDAL_RaiseDBAccess);
                eventLogDbDAL.RaiseDBAccessErrors += new RaiseDBAccessDelegate(eventLogDbDAL_RaiseDBAccessErrors);
                eventLogDbDAL.RaiseDBAccessInserts += new RaiseDBAccessDelegate(eventLogDbDAL_RaiseDBAccessInserts);
                eventLogDbDAL.RaiseDBAccessInsertDuplicates += new RaiseDBAccessDelegate(eventLogDbDAL_RaiseDBAccessInsertDuplicates);
                eventLogDbDAL.RaiseErrorMessage += new RaiseMessageDelegate(eventLogDbDAL_RaiseErrorMessage);
                eventLogDbDAL.RaiseInfoMessage += new RaiseMessageDelegate(eventLogDbDAL_RaiseInfoMessage);
                eventLogDbDAL.RaiseSyncInfoMessage += new RaiseMessageDelegate(eventLogDbDAL_RaiseSyncInfoMessage);
                eventLogDbDAL.DBAccessDurationInfoMessage += new DBAccessDurationDelegate(eventLogDbDAL_DBAccessDurationInfoMessage);
                #endregion

                #region Set up connection string
                if (Properties.Settings.Default.SQLServer.Length == 0)
                {
                    throw new Exception("Database connection is not set up!");
                }
                eventLogDbDAL.Server = Properties.Settings.Default.SQLServer;
                eventLogDbDAL.Database = Properties.Settings.Default.Database;
                eventLogDbDAL.SetConnection();
                #endregion
                //End of setting up data access object

                //Testing connection to ensure database availability.
                //If the machine is starting up SQL might not be available yet.
                //only retry 3 times and then raise exception
                if (!WaitForDbConn(3))
                {
                    throw new Exception("SQL server not available\r\n" + eventLogDbDAL.LastError);
                } 
                #endregion

                #region Check collector name
                if (Properties.Settings.Default.ColletorName == null || Properties.Settings.Default.ColletorName.Length == 0)
                {
                    throw new Exception("Collector name not specified in config file!");
                }
                thisCollector = eventLogDbDAL.GetCollectorByName(Properties.Settings.Default.ColletorName);
                if (thisCollector == null)
                {
                    throw new Exception("The collector name is not set up in the database!");
                } 
                #endregion

                #region Run maintenance
                if (eventLogDbDAL.GetMaintenanceRunner().ToUpper() == Properties.Settings.Default.ColletorName.ToUpper())
                {
                    //start maintenance job thread
                    EventLog.WriteEntry(ServiceEventSource, "This collector is set up to run system maintenance", EventLogEntryType.Information, (int)EventIDCodes.Maintenance);
                    new Thread(delegate()
                    {
                        RunMaintenance();
                    }
                    ).Start();

                    //Maintenance runner will also be responsible for history
                    //start Create history job thread
                    new Thread(delegate()
                    {
                        RunCreateHistory();
                    }
                    ).Start();
                } 
                #endregion

                #region Imports
                if (thisCollector.ImportsEnabled)
                {
                    //start data imports job thread
                    new Thread(delegate()
                    {
                        RunDataImport();
                    }
                    ).Start();
                } 
                #endregion

                //The following settings are only loaded on startup
                isRecyclingEnabled = eventLogDbDAL.GetThreadRecyclingEnabled(thisCollector.Name);
                recyclingCheckFrequency = eventLogDbDAL.GetRecyclingCheckFrequencyMinutes();

                sw.Stop();
                IncrementCounter(executeDurationCount, sw.ElapsedMilliseconds);

                //Master thread loop
                while (running)
                {
                    sw.Reset();
                    sw.Start();

                    pollingThreads = new List<Thread>();

                    //reset thread count performance counters
                    machineLogThreadCount.RawValue = 0;
                    machineLogActiveThreadCount.RawValue = 0;

                    //If database connection is down for some reason wait until it becomes available
                    WaitForDbConn(INFINITERETRIES);
                    machineLogs = eventLogDbDAL.GetMachineLogListByCollector(thisCollector.Name);
                    threadNo = 0;
                    foreach (MachineLogEntry machineLog in machineLogs)
                    {
                        #region Create thread for each
                        //create copies of the variables in memory
                        //if not done this way the new thread gets wrong 'copy' of variable sometime
                        //weird but true...
                        string machine = machineLog.Machine;
                        string logName = machineLog.LogName;
                        int threadNoIns = threadNo;

                        Thread scavengerThread = new Thread(delegate()
                        {
                            RunScavenger(machine, logName, threadNoIns);
                        }
                        );
                        pollingThreads.Add(scavengerThread);
                        scavengerThread.Start();

                        Thread.Sleep(250);
                        threadNo++; 
                        #endregion
                    }
                    sw.Stop();
                    IncrementCounter(executeDurationCount, sw.ElapsedMilliseconds);

                    if (isRecyclingEnabled)
                    {
                        #region Recycling
                        //routine for recycling
                        //  1. time set in database 
                        //  2. when time is reached the value in db must be reset to next interval.
                        //  3. value must only be checked something like every 5 minutes
                        SetThreadCounterActiveState(isActiveIndicator, false);
                        WaitForNextRecycleTime(recyclingCheckFrequency);
                        SetThreadCounterActiveState(isActiveIndicator, true);

                        sw.Reset();
                        sw.Start();
                        //If for some reason the database is unavailable during recycling we wait till it gets back up
                        WaitForDbConn(INFINITERETRIES);

                        if (running)
                        {
                            EventLog.WriteEntry(ServiceEventSource, "Recycling all polling threads", EventLogEntryType.Information, (int)EventIDCodes.RecyclingEvent);
                            recycling = true;
                            //give 30 sec time for running threads to stop
                            Thread.Sleep(30000);
                            recycling = false;
                            //Abort any running threads left
                            foreach (Thread poller in pollingThreads)
                            {
                                if (poller.IsAlive)
                                {
                                    poller.Abort();
                                    poller.Join(waitingTimeDecrement);
                                }
                            }
                        }
                        sw.Stop();
                        IncrementCounter(executeDurationCount, sw.ElapsedMilliseconds);
                        #endregion
                    }
                    else
                    {
                        #region No recycling
                        SetThreadCounterActiveState(isActiveIndicator, false);
                        while (running)
                        {
                            Thread.Sleep(waitingTimeDecrement);
                        } 
                        #endregion
                    }
                    SetThreadCounterActiveState(isActiveIndicator, true);
                }
                SetThreadCounterActiveState(isActiveIndicator, false);
                ClosePerformanceCounters();
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry(ServiceEventSource, ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.GeneralError);
                HaltService("Stopping service automatically because of critical problem.");
            }
        }        
        #endregion

        #region RunScavenger
        /// <summary>
        /// Polling method to retrieve event logs and save into database
        /// </summary>
        /// <param name="machine">Name of the computer</param>
        /// <param name="logName">The event log name e.g. Application</param>
        /// <param name="threadNo">The thread number</param>
        private static void RunScavenger(string machine, string logName, int threadNo)
        {
            DateTime lastPollTime;
            string lastStep = "Start RunScavenger";
            MachineLogEntry machineLogEntry = null;
            int pollFrequency = 2;
            bool hostOffline = false;
            DateTime? pollFilterLastChanged = null;
            PollFilter pollFilter = null;
            PerformanceCounter executeDurationCount = null;
            PerformanceCounter isActiveIndicator = null;
            Stopwatch sw = new Stopwatch();

            //initialize duration performance counter
            try
            {
                executeDurationCount = InitializePerfCounterInstance(perfCategoryThreads, executionDurationTimeCounterName, machine + "\\" + logName);
                isActiveIndicator = InitializePerfCounterInstance(perfCategoryThreads, isActiveIndicatorCounterName, machine + "\\" + logName);
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry(ServiceEventSource, "There was an error initializing the '" + perfCategoryThreads + "' performance counters for the thread '" + machine + "\\" + logName + "'.\r\n" + ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.GeneralError);
            }
            
            //Initialize machineLogEntry object
            try
            {
                SetThreadCounterActiveState(isActiveIndicator, true);
                sw.Start();    
                machineLogEntry = eventLogDbDAL.GetMachineLogEntry(machine, logName);
                pollFilterLastChanged = machineLogEntry.PollFilterLastChanged;
                if (machineLogEntry.PollFilterID > 0)
                {
                    pollFilter = eventLogDbDAL.GetPollFilterByID(machineLogEntry.PollFilterID);
                }
                sw.Stop();
                IncrementCounter(executeDurationCount, sw.ElapsedMilliseconds);
            }
            catch { }

            reportThreadToMaster.WaitOne();
#if DEBUG
            string startingInfo = "";
            if (machineLogEntry == null || machineLogEntry.MachineLogID == -1)
                startingInfo = string.Format("Starting scavenger thread : {0}\\{1}, Thread Id:{2}", machine, logName, threadNo);
            else
                startingInfo = string.Format("Starting scavenger thread : {0}\\{1}, Thread Id:{2}\r\nCurrent state: {3}", machine, logName, threadNo, (machineLogEntry.Enabled ? "Enabled" : "Not enabled"));
            if (pollFilter != null)
                startingInfo += string.Format("\r\nUsing polling filter: {0}", pollFilter.Name);
            EventLog.WriteEntry(ServiceEventSource, startingInfo, EventLogEntryType.Information, (int)EventIDCodes.StartThread);
#endif
            reportThreadToMaster.ReleaseMutex();

            IncrementCounter(machineLogThreadCount);
            IncrementCounter(machineLogActiveThreadCount);

            while (running && !recycling)
            {
                try
                {
                    sw.Reset();
                    sw.Start();

                    lastStep = "Check if network is up";
                    if (!PingHelper.IsOffline()) //only bother if we're online
                    {
                        //Update MachineLog details to find
                        //1. is the machinelog enabled for polling
                        //2. the oldest existing entry for this log in the database
                        lastStep = "Retrieve latest MachineLog entry details";
                        machineLogEntry = eventLogDbDAL.GetMachineLogEntry(machine, logName);
                        //test if pollfilter has changed
                        if (machineLogEntry.PollFilterID > 0)
                        {
                            if (machineLogEntry.PollFilterLastChanged.HasValue && (machineLogEntry.PollFilterLastChanged > pollFilterLastChanged))
                            {
                                lastStep = "Load Polling filter settings";
                                pollFilterLastChanged = machineLogEntry.PollFilterLastChanged;
                                pollFilter = eventLogDbDAL.GetPollFilterByID(machineLogEntry.PollFilterID);
                            }
                        }
                        else //if the filter has been removed
                            pollFilter = null;

                        if (machineLogEntry.MachineLogID > 0)
                        {
                            pollFrequency = machineLogEntry.PollFrequency;
                            if (machineLogEntry.Enabled && machineLogEntry.Collector.ToUpper() == Properties.Settings.Default.ColletorName.ToUpper())
                            {
#if DEBUG
                                Trace.WriteLine("Thread:" + threadNo.ToString() + ",Polling:" + machine + "\\" + logName + ", Tm:" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
#endif

                                lastStep = "Retrieve last polling time";
                                lastPollTime = machineLogEntry.LastPollTime;

                                lastStep = "Make sure host/machine is available on the network";
                                //Make sure the machine is on the network
                                if (PingHelper.CanPingHost(machine))
                                {
                                    //if the host was offline before report that is available again.
                                    if (hostOffline)
                                    {
                                        EventLog.WriteEntry(ServiceEventSource, "The host/machine " + machine + " is available again", EventLogEntryType.Information, (int)EventIDCodes.Network);
                                    }
                                    hostOffline = false;
                                    #region Poll Eventlog and write to database
                                    lastStep = "Retrieve eventlog";
                                    PollMachineEventLog(lastPollTime, machine, logName, threadNo, pollFilter);
                                    #endregion
                                }
                                else if (!hostOffline)
                                {
                                    hostOffline = true;
                                    lastStep = "Retrieve reason ping failed";
                                    string reason = PingHelper.GetPingHostErrorReason(machine);
                                    LogError("The host/machine " + machine + " is not available!\r\nReason:" + reason, EventIDCodes.Network, lastStep, machine, logName, threadNo);
                                }
                            } //if (machineLogEntry.Enabled)
                        }//if (machineLogEntry != null)
                    } //if (PingHelper.IsOffline())
                    sw.Stop();
                    IncrementCounter(executeDurationCount, sw.ElapsedMilliseconds);
                }
                catch (Exception ex) //handling all other exceptions within this thread
                {
                    LogError(ex.ToString() + "\r\n" + ex.GetType().ToString(), EventIDCodes.GeneralError, lastStep, machine, logName, threadNo);
                }

                //finishing/cleaning up
                if (machineLogEntry != null)
                {
                    if (!machineLogEntry.Enabled)
                        DecrementCounter(machineLogActiveThreadCount);
                }
                SetThreadCounterActiveState(isActiveIndicator, false);
                //go to sleep until next poll event
                WaitForTimeOrExit(pollFrequency * 60000, waitingTimeDecrement);
                SetThreadCounterActiveState(isActiveIndicator, true);
                if (machineLogEntry != null)
                {
                    if (!machineLogEntry.Enabled)
                        IncrementCounter(machineLogActiveThreadCount);
                }
            } //end while
#if DEBUG
            EventLog.WriteEntry(ServiceEventSource, "Stopping scavenger thread : " + machine + "\\" + logName + ", Thread:" + threadNo.ToString(), EventLogEntryType.Information, (int)EventIDCodes.StopThread);
#endif
            DecrementCounter(machineLogThreadCount);
            DecrementCounter(machineLogActiveThreadCount);
            SetThreadCounterActiveState(isActiveIndicator, false);
        }
        private static List<EventLogEntry> GetNewEventLogEntries(
            DateTime untilTime, 
            string machine, 
            string logName,
            int threadNo,
            PollFilter currentPollFilter)
        {
            //new method with different flow
            //1. Get last entry datetime for log from db
            //2. Create local (temporary) list<> 
            //3. open log from machine
            //4. Read entries from back until entry date older than last entry date
            //5.   Append to local temporary list
            string lastStep = "Read Eventlog entries of " + machine + "\\" + logName;
            int indexErr = 0;
            EventLogEntry entry2Read = null;
            List<EventLogEntry> eventBufferList = new List<EventLogEntry>();
            try
            {
                using (EventLog log2Read = new EventLog(logName, machine))
                {
                    lastStep = "Getting index of oldest entry";
                    int index = log2Read.Entries.Count - 1;
                    //the following loop is to eliminate the event where entries have been 
                    //truncated since the eventlog has been opened causing the 
                    //'Index x is out of bounds' error
                    #region Get biggest index not causing an 'Index out of bounds' error.
                    while (index > -1)
                    {
                        try
                        {
                            entry2Read = log2Read.Entries[index];
                            break;
                        }
                        catch (System.ArgumentException aex)
                        {
                            if (aex.Message.Contains("is out of bounds")) //like in Index xxx is out of bounds.
                            {
                                index--;
                                if (!running || paused || recycling)
                                {
                                    return new List<EventLogEntry>();
                                }
                            }
                            else
                                throw;
                        }
                    } 
                    #endregion
                    
                    for (int i = index; i >= 0; i--)
                    {
                        try
                        {
                            lastStep = "Reading entry " + i.ToString();
                            entry2Read = log2Read.Entries[i];
                            lastStep = "Check if it is newer than existing entries";
                            if (entry2Read.TimeGenerated > untilTime)
                            {
                                if (currentPollFilter == null)
                                    eventBufferList.Insert(0, entry2Read);
                                else if (currentPollFilter.IsEventLogEntryTypeInFilter(entry2Read.EntryType)
                                        && (
                                                (!currentPollFilter.UseListFilters) || //don't bother if there are no lists/text
                                                currentPollFilter.IsInFilter(entry2Read.Message, (int)(entry2Read.InstanceId & 65535), entry2Read.Source))
                                            )
                                    eventBufferList.Insert(0, entry2Read);
                            }
                            else
                            {
                                break;
                            }
                            indexErr = 0;
                        }
                        catch (System.ArgumentException aex)
                        {
                            if (aex.Message.Contains("is out of bounds")) //like in Index xxx is out of bounds.
                            {
                                indexErr++;
                                if (indexErr > 10)
                                {
                                    LogError(aex.Message, EventIDCodes.ReadEventLogError, lastStep, machine, logName, threadNo);
                                    break;
                                }
                            }
                            else
                                throw;
                        }
                        lastStep = "Check if thread is still running";
                        //if the service is stopping, paused or recycling stop processing
                        if (!running || paused || recycling)
                        {
                            eventBufferList = new List<EventLogEntry>();
                            break;
                        }
                    }
                }
            }
            catch (System.IO.IOException)
            {
                try
                {
                    lastStep = "Trying to connect using full dns name";
                    string fullDNSName = System.Net.Dns.GetHostEntry(machine).HostName;
                    if (fullDNSName != machine)
                        eventBufferList = GetNewEventLogEntries(untilTime, fullDNSName, logName, threadNo, currentPollFilter);
                    else
                    {
                        IncrementCounter(errorsPerSecond);
                        LogError("The network path was not found (using full dns name)", EventIDCodes.ReadEventLogError, entry2Read, lastStep, machine, logName, threadNo);
                    }
                }
                catch (Exception innerex)
                {
                    IncrementCounter(errorsPerSecond);
                    LogError(innerex.Message + "\r\n" + innerex.GetType().ToString(), EventIDCodes.ReadEventLogError, entry2Read, lastStep, machine, logName, threadNo);
                }
            }
            catch (Exception ex)
            {
                IncrementCounter(errorsPerSecond);
                LogError(ex.Message + "\r\n" + ex.GetType().ToString(), EventIDCodes.ReadEventLogError, entry2Read, lastStep, machine, logName, threadNo);
            }
            return eventBufferList;
        }
        private static void PollMachineEventLog(
                DateTime previousLastEntryTime, 
                string machine, 
                string logName, 
                int threadNo ,
                PollFilter currentPollFilter 
            )
        {
            string lastStep = "Polling Eventlog of " + machine + "\\" + logName;
            int batchSize = 100;
            int batchTimeUpdaterCounter = 0;
            int connectionErrors = 0;
            DateTime lastEntryDateTime = previousLastEntryTime;
            DateTime pollStartTime = DateTime.Now;
            List<EventLogEntry> eventBufferList;
            try
            {                
                //Retrieve only 'new' eventlog entries in oldest to newest order
                eventBufferList = GetNewEventLogEntries(previousLastEntryTime, machine, logName, threadNo, currentPollFilter);
                
                lastStep = "Writing Eventlog entries to database";
                if (running && !paused && !recycling)
                {
                    batchTimeUpdaterCounter = 0;

                    if (Properties.Settings.Default.UseBatchInserts)
                    {
                        List<EventLogEntry> batch = null;
                        batchSize = Properties.Settings.Default.InsertBatchSize;
                        lastStep = "Inserting event log entry batches";
                        while (eventBufferList.Count > batchSize)
                        {
                            lastStep = "Getting new batch of event log entries";
                            batch = eventBufferList.Take(batchSize).ToList();
                            lastStep = "Inserting batch of event log entries";
                            XmlBatchInsert(batch, previousLastEntryTime, machine, logName, threadNo, currentPollFilter);
                            lastStep = "Getting newest entry time from batch";
                            lastEntryDateTime = batch.Max(en => en.TimeGenerated);
                            lastStep = "Removing batch of event log entries";
                            eventBufferList.RemoveRange(0, batchSize);
                            lastStep = "Writing last polling time";
                            eventLogDbDAL.SetLastMachineLogPollingTime(machine, logName, lastEntryDateTime);

                            //if the service is stopping, paused or recycling stop processing
                            if (!running || paused || recycling)
                            {
                                return;
                            }
                        }
                        if (eventBufferList.Count > 0)
                        {
                            lastStep = "Inserting batch of event log entries";
                            XmlBatchInsert(eventBufferList, previousLastEntryTime, machine, logName, threadNo, currentPollFilter);
                            lastStep = "Getting newest entry time from batch";
                            lastEntryDateTime = eventBufferList.Max(en => en.TimeGenerated);
                            lastStep = "Writing last polling time";
                            eventLogDbDAL.SetLastMachineLogPollingTime(machine, logName, lastEntryDateTime);
                        }
                    }
                    else
                    {
                        foreach (EventLogEntry entry2Read in eventBufferList)
                        {
                            batchTimeUpdaterCounter++;
                            lastStep = "Reading Event ID";
                            int eventId = (int)(entry2Read.InstanceId & 65535);
                            lastStep = "Attempting to write entry to database";
                            try
                            {
                                eventLogDbDAL.InsertEventLogEntry(
                                    machine,
                                    logName,
                                    entry2Read.Index,
                                    eventId,
                                    entry2Read.TimeGenerated,
                                    entry2Read.Source,
                                    entry2Read.EntryType.ToString(),
                                    Encoding.UTF8.GetString(entry2Read.Data, 0, entry2Read.Data.Length),
                                    entry2Read.Message,
                                    entry2Read.Category,
                                    entry2Read.UserName);
                                IncrementCounter(eventEntriesPerSecond);
                                connectionErrors = 0;
                            }
                            catch (System.ArgumentOutOfRangeException)
                            {
                                IncrementCounter(errorsPerSecond); //inc perf counter but continue
                            }
                            catch (System.InvalidOperationException ioEx)
                            {
                                IncrementCounter(errorsPerSecond);
                                if (ioEx.Message.Contains("Internal connection fatal error."))
                                {
                                    connectionErrors++;
                                    if (connectionErrors > 2) //3 retries
                                        throw;
                                }
                                else
                                    throw;
                            }
                            catch
                            {
                                IncrementCounter(errorsPerSecond);
                                throw;
                            }

                            //Set last entry datetime to current entry in case service stops/fails etc. to continue from this point if restarted
                            lastEntryDateTime = entry2Read.TimeGenerated;

                            //if batchsize is reached update machinelog
                            if (batchTimeUpdaterCounter > batchSize)
                            {
                                batchTimeUpdaterCounter = 0;
                                lastStep = "Writing last polling time";
                                eventLogDbDAL.SetLastMachineLogPollingTime(machine, logName, lastEntryDateTime);
                            }

                            //if the service is stopping, paused or recycling stop processing
                            if (!running || paused || recycling)
                                break;
                        }
                    }
                    if (running)
                    {
                        lastStep = "Writing last polling time";
                        if (currentPollFilter == null || (eventBufferList != null && eventBufferList.Count > 0))
                            eventLogDbDAL.SetLastMachineLogPollingTime(machine, logName, lastEntryDateTime);
                        else //if there were no events matching the filter during that time then set to time started polling
                            //otherwise the service has to struggle through all the filtered events again.
                            eventLogDbDAL.SetLastMachineLogPollingTime(machine, logName, pollStartTime);

                    }
                }
            }
            catch (System.Data.SqlClient.SqlException sqlex)
            {
                LogError(sqlex.ToString(), EventIDCodes.DatabaseError, lastStep, machine, logName, threadNo);
            }
            catch (System.ComponentModel.Win32Exception win32ex)
            {
                LogError(win32ex.ToString(), EventIDCodes.RemoteProcedure, lastStep, machine, logName, threadNo);
            }
            catch (System.InvalidOperationException invalex)
            {
                LogError(invalex.ToString(), EventIDCodes.InvalidOperation, lastStep, machine, logName, threadNo);
            }
            catch (System.IO.IOException ioex)
            {
                LogError(ioex.ToString(), EventIDCodes.IOError, lastStep, machine, logName, threadNo);
            }
            catch (System.Security.SecurityException scex)
            {
                LogError(scex.ToString(), EventIDCodes.SecurityError, lastStep, machine, logName, threadNo);
            }
            catch (Exception ex) //reporting all other exceptions within this thread
            {
                LogError(ex.ToString() + "\r\n" + ex.GetType().ToString(), EventIDCodes.GeneralError, lastStep, machine, logName, threadNo);
            }
        }
        /// <summary>
        /// This method will insert a batch of event log entries into the database using the new
        /// InsertEventLogEntries stored procedure. It will pass one xml parameter so it does not
        /// have to make multiple calls to the database for inserting each entry on its own
        /// </summary>
        /// <param name="iEnumerable"></param>
        /// <param name="previousLastEntryTime"></param>
        /// <param name="machine"></param>
        /// <param name="logName"></param>
        /// <param name="threadNo"></param>
        /// <param name="currentPollFilter"></param>
        private static void XmlBatchInsert(IEnumerable<EventLogEntry> eventEntries, 
            DateTime previousLastEntryTime, string machine, string logName, int threadNo, PollFilter currentPollFilter)
        {
            try
            {
                System.IO.MemoryStream outstream = new System.IO.MemoryStream();
                using (System.Xml.XmlTextWriter wrt = new System.Xml.XmlTextWriter(outstream, Encoding.UTF8))
                {
                    wrt.Formatting = System.Xml.Formatting.Indented;
                    wrt.Indentation = 1;
                    wrt.IndentChar = ' ';
                    wrt.WriteStartDocument();
                    wrt.WriteStartElement("rs");

                    foreach (EventLogEntry ele in eventEntries)
                    {
                        wrt.WriteStartElement("r");
                        wrt.WriteAttributeString("cn", machine);
                        wrt.WriteAttributeString("l", logName);
                        wrt.WriteAttributeString("r", ele.Index.ToString());
                        int eventId = (int)(ele.InstanceId & 65535);
                        wrt.WriteAttributeString("e", eventId.ToString());
                        wrt.WriteAttributeString("t", ele.TimeGenerated.ToString("yyyy-MM-ddTHH:mm:ss"));
                        wrt.WriteAttributeString("s", ele.Source);
                        wrt.WriteAttributeString("et", ele.EntryType.ToString().ToUpper().Substring(0, 1));
                        wrt.WriteAttributeString("c", N(ele.Category));
                        wrt.WriteAttributeString("u", N(ele.UserName));
                        string message = ele.Message;
                        string strings = Encoding.UTF8.GetString(ele.Data, 0, ele.Data.Length);
                        if (message.Length == 0 && strings != null && strings.Length > 0)
                            if (!strings.Contains('�'))
                                message = strings;
                        message = message.Replace("\r\n", "\\r\\n");
                        wrt.WriteAttributeString("m", message);
                        wrt.WriteEndElement(); //r
                    }

                    wrt.WriteEndElement(); //rs
                    wrt.WriteEndDocument();
                    wrt.Flush();

                    outstream.Position = 0;
                    string infoResult = eventLogDbDAL.InsertEventLogEntries(outstream);
                    if (infoResult.Length > 0 && Properties.Settings.Default.ShowDuplicateWarnings)
                    {
                        EventLog.WriteEntry(ServiceEventSource, 
                            "Not all entries could be inserted into the database!\r\n" + infoResult +
                            "\r\nFor '" + machine + "\\" + logName + "'", EventLogEntryType.Warning, 
                            (int)EventIDCodes.RepeatError);
                    }
                }                
                IncrementCounter(eventEntriesPerSecond, eventEntries.Count());
            }
            catch (Exception ex)
            {
                IncrementCounter(errorsPerSecond);
            }
        }
        #endregion

        #region Maintenance thread
        private static void RunMaintenance()
        {
            int maintenanceFrequencyMinutes = 5; //default
            PerformanceCounter executeDurationCount = null;
            PerformanceCounter isActiveIndicator = null;
            Stopwatch sw = new Stopwatch();
            try
            {
                executeDurationCount = InitializePerfCounterInstance(perfCategoryThreads, executionDurationTimeCounterName, "Maintenance");
                isActiveIndicator = InitializePerfCounterInstance(perfCategoryThreads, isActiveIndicatorCounterName, "Maintenance");
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry(ServiceEventSource, "There was an error initializing the '" + perfCategoryThreads + "' performance counters for the maintenance thread.\r\n" + ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.Maintenance);
            }

            while (running)
            {
                string lastStep = "Run Maintenance";
                
                try
                {
                    SetThreadCounterActiveState(isActiveIndicator, true);
                    sw.Reset();
                    sw.Start();
                    WaitForDbConn(INFINITERETRIES);                    
                    if (eventLogDbDAL.IsMaintenanceEnabled())
                    {
                        eventLogDbDAL.RunMaintenance(eventLogDbDAL.GetMaintenanceTimeOutSeconds());
                        eventLogDbDAL.SetMaintenanceLastRunTime();
                    }
                    else
                        EventLog.WriteEntry(ServiceEventSource, "Maintenance disabled", EventLogEntryType.Information, (int)EventIDCodes.Maintenance);
                    maintenanceFrequencyMinutes = eventLogDbDAL.GetMaintenanceFrequencyMinutes();
                    sw.Stop();
                    IncrementCounter(executeDurationCount, sw.ElapsedMilliseconds);
                    SetThreadCounterActiveState(isActiveIndicator, false);
                }
                catch (Exception ex) //handling all exceptions within this thread
                {
                    EventLog.WriteEntry(ServiceEventSource, lastStep + "\r\n" + ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.Maintenance);
                }
                
                WaitForTimeOrExit(maintenanceFrequencyMinutes * 60000, waitingTimeDecrement);
            }            
        }
        #endregion

        #region Create history thread
        private static void RunCreateHistory()
        {
            PerformanceCounter executeDurationCount = null;
            PerformanceCounter isActiveIndicator = null;
            Stopwatch sw = new Stopwatch();
            try
            {
                executeDurationCount = InitializePerfCounterInstance(perfCategoryThreads, executionDurationTimeCounterName, "Create history");
                isActiveIndicator = InitializePerfCounterInstance(perfCategoryThreads, isActiveIndicatorCounterName, "Create history");
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry(ServiceEventSource, "There was an error initializing the '" + perfCategoryThreads + "' performance counters for the create history thread.\r\n" + ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.Maintenance);
            }
            while (running)
            {
                try
                {
                    SetThreadCounterActiveState(isActiveIndicator, true);
                    sw.Reset();
                    sw.Start();
                    //If for some reason the database is unavailable during recycling we wait till it gets back up
                    if (WaitForDbConn(INFINITERETRIES))
                    {
                        if (running)
                        {
                            if (eventLogDbDAL.GetRunCreateHistoryEnabled())
                            {
                                EventLog.WriteEntry(ServiceEventSource, "Running create history job", EventLogEntryType.Information, (int)EventIDCodes.Maintenance);
                                eventLogDbDAL.RunCreateHistory();
                                EventLog.WriteEntry(ServiceEventSource, "Create history job done", EventLogEntryType.Information, (int)EventIDCodes.Maintenance);
                            }
                        }
                    }
                    sw.Stop();
                    IncrementCounter(executeDurationCount, sw.ElapsedMilliseconds);
                    SetThreadCounterActiveState(isActiveIndicator, false);
                    WaitForNextCreateHistoryTime();
                }
                catch (Exception ex)
                {
                    EventLog.WriteEntry(ServiceEventSource, "Create history job failed\r\n" + ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.Maintenance);
                }
            }
        }
        #endregion

        #region Data imports from other event scavenger databases
        private static void RunDataImport()
        {
            PerformanceCounter executeDurationCount = null;
            PerformanceCounter isActiveIndicator = null;
            Stopwatch sw = new Stopwatch();
            try
            {
                executeDurationCount = InitializePerfCounterInstance(perfCategoryThreads, executionDurationTimeCounterName, "Data import");
                isActiveIndicator = InitializePerfCounterInstance(perfCategoryThreads, isActiveIndicatorCounterName, "Data import");
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry(ServiceEventSource, "There was an error initializing the '" + perfCategoryThreads + "' performance counters for the create history thread.\r\n" + ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.Maintenance);
            }
            string lastStep = "Run Imports";
            while (running)
            {
                SetThreadCounterActiveState(isActiveIndicator, true);
                sw.Reset();
                sw.Start();
                try
                {
                    lastStep = "Wait for database";
                    WaitForDbConn(INFINITERETRIES);
                    lastStep = "Get list of logs to import";
                    foreach (ImportForeignLog importlog in eventLogDbDAL.SelectImportLogsByCollector(thisCollector.Id))
                    {
                        try
                        {
                            List<MachineLogEntry> logs = new List<MachineLogEntry>();
                            logs.Add(new MachineLogEntry(importlog.Machine, importlog.LogName));
                            lastStep = "Import - " + importlog.SQLServer + "\\" + importlog.Database;
                            eventLogDbDAL.ImportLog(importlog.SQLServer, importlog.Database, logs);
                        }
                        catch (System.Data.SqlClient.SqlException sqlex)
                        {
                            EventLog.WriteEntry(ServiceEventSource, lastStep + "\r\n" + sqlex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.DataImport);
                        }
                    }                   
                }
                catch (Exception ex) //handling all exceptions within this thread
                {
                    EventLog.WriteEntry(ServiceEventSource, lastStep + "\r\n" + ex.ToString(), EventLogEntryType.Error, (int)EventIDCodes.DataImport);
                }
                sw.Stop();
                IncrementCounter(executeDurationCount, sw.ElapsedMilliseconds);
                SetThreadCounterActiveState(isActiveIndicator, false);
                WaitForTimeOrExit(thisCollector.ImportFrequency * 60000, waitingTimeDecrement);
            }
        }
        #endregion

        #region Waiting helper methods
        /// <summary>
        ///Waiting in increments of 'decrementBy' and also checks if service is still running
        ///When paused it just waits indefinitely
        /// </summary>
        private static void WaitForTimeOrExit(int nextWaitInterval, int decrementBy)
        {
            int waitTimeRemaining;
            if (!running)
                return;
            try
            {
                if ((nextWaitInterval <= decrementBy) && (nextWaitInterval > 0))
                {
                    Thread.Sleep(nextWaitInterval);
                }
                else
                {
                    waitTimeRemaining = nextWaitInterval;
                    while (running && (waitTimeRemaining > 0) && (!recycling))
                    {
                        if (waitTimeRemaining <= decrementBy)
                        {
                            waitTimeRemaining = 0;
                        }
                        else
                        {
                            waitTimeRemaining -= decrementBy;
                        }
                        if (decrementBy > 0)
                        {
                            Thread.Sleep(decrementBy);
                        }
                    }
                }
                while (paused && running && (!recycling))
                {
                    Thread.Sleep(decrementBy);
                }
            }
            catch (ThreadAbortException excabort)
            {
                EventLog.WriteEntry(ServiceEventSource, "Service aborting\r\n" + excabort.ToString(), EventLogEntryType.Error);
                throw;
            }
        }
        /// <summary>
        /// Similar to WaitForTimeOrExit but waits for specific datetime as specified in database
        /// </summary>
        private static void WaitForNextRecycleTime(int recyclingCheckFrequency)
        {
            int nextWaitInterval = recyclingCheckFrequency * 60000;
            if (!running)
                return;
            try
            {
                eventLogDbDAL.SetNextRecycleTime(Properties.Settings.Default.ColletorName);

                DateTime nextRecycleTime = eventLogDbDAL.GetNextRecycleTime(Properties.Settings.Default.ColletorName);
                while (running && (nextRecycleTime > DateTime.Now))
                {
                    WaitForTimeOrExit(nextWaitInterval, waitingTimeDecrement);
                    if (running)
                        nextRecycleTime = eventLogDbDAL.GetNextRecycleTime(Properties.Settings.Default.ColletorName);
                }
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry(ServiceEventSource, "WaitForNextRecycleTime:" + ex.ToString(), EventLogEntryType.Error);
                WaitForTimeOrExit(nextWaitInterval, waitingTimeDecrement);
            }
        }
        /// <summary>
        /// Similar to WaitForTimeOrExit but waits for specific 'create history' datetime as specified in database
        /// </summary>
        private static void WaitForNextCreateHistoryTime()
        {
            int nextWaitInterval = 60000; //one minute
            if (!running)
                return;
            try
            {
                eventLogDbDAL.SetNextCreateHistoryTime();

                DateTime nextCreateHistoryTime = eventLogDbDAL.GetNextCreateHistoryTime();
                while (running && (nextCreateHistoryTime > DateTime.Now))
                {
                    WaitForTimeOrExit(nextWaitInterval, waitingTimeDecrement);
                    if (running)
                        nextCreateHistoryTime = eventLogDbDAL.GetNextCreateHistoryTime();
                }
            }
            catch (Exception ex)
            {
                EventLog.WriteEntry(ServiceEventSource, "WaitForNextCreateHistoryTime:" + ex.ToString(), EventLogEntryType.Error);
                WaitForTimeOrExit(nextWaitInterval, waitingTimeDecrement);
            }
        }
        /// <summary>
        /// Test for valid database connection or waits for connection to become available
        /// </summary>
        /// <param name="retryTimes">Number of retries</param>
        /// <returns></returns>
        private static bool WaitForDbConn(int retryTimes)
        {
            bool success = true;
            while (running && (!eventLogDbDAL.TestConnection()) &&
                   ((retryTimes == INFINITERETRIES) || (retryTimes > 0)))
            {
                EventLog.WriteEntry(ServiceEventSource, "SQL server connection failed. This could be because SQL server is not started or not available yet. Waiting 1 minute to retry connection.", EventLogEntryType.Warning, (int)EventIDCodes.ConnectionError);
                WaitForTimeOrExit(60000, waitingTimeDecrement); //wait 1 minute and try again
                if (retryTimes > 0)
                    retryTimes--;
            }
            if (running && (retryTimes == 0)) //ran out of retries
                success = false;

            return success;
        }
        #endregion

        #region Error logging
        private static void LogError(string errorMessage, EventIDCodes errorCode, string lastStep, string machine, string logName, int threadNo)
        {
            try
            {
                EventLog.WriteEntry(ServiceEventSource, lastStep + "\r\n" +
                                   "Machine:" + machine + "\r\n" +
                                   "Log:" + logName + "\r\n" +
                                   "Thread No:" + threadNo.ToString() + "\r\n" +
                                   errorMessage, EventLogEntryType.Error, (int)errorCode);
            }
            catch { } //Ignore further errors here. This is only an attempt for more info.
        }
        private static void LogError(string errorMessage, EventIDCodes errorCode, EventLogEntry entry2Read, string lastStep, string machine, string logName, int threadNo)
        {
            try
            {
                EventLog.WriteEntry(ServiceEventSource, lastStep + "\r\n" +
                                   "Machine:" + machine + "\r\n" +
                                   "Log:" + logName + "\r\n" +
                                   "Thread No:" + threadNo.ToString() + "\r\n" +
                                   errorMessage, EventLogEntryType.Error, (int)errorCode);

                if (entry2Read != null)
                {
                    int eventId = (int)(entry2Read.InstanceId & 65535);
                    string lastEntry = "Last step before: " + lastStep +
                            "\r\nLast entry processed\r\nTime:" + entry2Read.TimeGenerated.ToString() +
                            "\r\nMachine:" + machine +
                            "\r\nLog:" + logName +
                            "\r\nSource:" + entry2Read.Source +
                            "\r\nType:" + entry2Read.EntryType.ToString() +
                            "\r\nEventID:" + eventId.ToString() +
                            "\r\nMessage:" + entry2Read.Message;
                    EventLog.WriteEntry(ServiceEventSource, lastEntry, EventLogEntryType.Warning, (int)EventIDCodes.RepeatError);
                }
            }
            catch { } //Ignore further errors here. This is only an attempt for more info.
        }
        private static void eventLogDbDAL_RaiseErrorMessage(string message)
        {
            try
            {
                EventLog.WriteEntry(ServiceEventSource, message, EventLogEntryType.Error, (int)EventIDCodes.GeneralError);
            }
            catch { }
        }
        private static void eventLogDbDAL_RaiseInfoMessage(string message)
        {
            try
            {
                EventLog.WriteEntry(ServiceEventSource, message, EventLogEntryType.Information, (int)EventIDCodes.SystemEvents);
            }
            catch { }
        }
        private static void eventLogDbDAL_RaiseSyncInfoMessage(string message)
        {
            if (Properties.Settings.Default.LogSyncInfoMessages)
            {
                try
                {
                    EventLog.WriteEntry(ServiceEventSource, message, EventLogEntryType.Information, (int)EventIDCodes.DataImport);
                }
                catch { }
            }
        }
        private static void eventLogDbDAL_DBAccessDurationInfoMessage(long milliseconds)
        {
            IncrementCounter(dbAccessDurationMilliseconds, milliseconds);
        }
        #endregion

        #region Performance Counters
        private static void eventLogDbDAL_RaiseDBAccess()
        {
            IncrementCounter(dbAccessPerSecond);
        }
        private static void eventLogDbDAL_RaiseDBAccessErrors()
        {
            IncrementCounter(errorsPerSecond);
        }
        private static void eventLogDbDAL_RaiseDBAccessInserts()
        {
            IncrementCounter(dbInsertsPerSecond);
        }
        private static void eventLogDbDAL_RaiseDBAccessInsertDuplicates()
        {
            IncrementCounter(dbDuplicateInsertsPerSecond);
        }
        private static void InitPerformanceCounters()
        {
            try
            {
                if (Properties.Settings.Default.RecreatePerfCountersOnStart)
                {
                    //Single instance stuff
                    if (PerformanceCounterCategory.Exists(ServiceEventSource))
                        PerformanceCounterCategory.Delete(ServiceEventSource);
                    //Multi instance performance counters
                    if (PerformanceCounterCategory.Exists(perfCategoryThreads))
                        PerformanceCounterCategory.Delete(perfCategoryThreads);
                }

                if (!PerformanceCounterCategory.Exists(ServiceEventSource))
                {
                    CounterCreationData[] counterCreationData = new CounterCreationData[]
                    {
                        new CounterCreationData("Total event logs", "Total number of event logs monitored", PerformanceCounterType.NumberOfItems32),
                        new CounterCreationData("Active event logs", "Total active number of event logs monitored", PerformanceCounterType.NumberOfItems32),
                        new CounterCreationData("Event entries/Sec", "Event entries read per second", PerformanceCounterType.RateOfCountsPerSecond32),
                        new CounterCreationData("Errors/Sec", "Total errors per second", PerformanceCounterType.RateOfCountsPerSecond32),
                        new CounterCreationData("Database accesses/Sec", "Total database accesses per second", PerformanceCounterType.RateOfCountsPerSecond32),
                        new CounterCreationData("Database inserts/Sec", "Database Inserts per second", PerformanceCounterType.RateOfCountsPerSecond32),
                        new CounterCreationData("Database duplicate inserts/Sec", "Database duplicate inserts per second", PerformanceCounterType.RateOfCountsPerSecond32)
                        
                    };
                    PerformanceCounterCategory.Create(ServiceEventSource, "Event Scavenger Service", PerformanceCounterCategoryType.SingleInstance, new CounterCreationDataCollection(counterCreationData));
                }

                if (!PerformanceCounterCategory.Exists(perfCategoryThreads))
                {
                    CounterCreationData[] threadCounterCreationData = new CounterCreationData[]
                    {
                        new CounterCreationData(executionDurationTimeCounterName, "Execution Duration time in milliseconds", PerformanceCounterType.RateOfCountsPerSecond32),
                        new CounterCreationData(isActiveIndicatorCounterName, "Thread activity indicator", PerformanceCounterType.NumberOfItems32)
                    };
                    PerformanceCounterCategory.Create(perfCategoryThreads, perfCategoryThreads, PerformanceCounterCategoryType.MultiInstance, new CounterCreationDataCollection(threadCounterCreationData));
                }
                Thread.Sleep(1000);
            }
            catch (Exception ex)
            {
                LogError(ex.ToString(), EventIDCodes.SystemEvents, null, "Creating performance counter category", System.Net.Dns.GetHostName(), "N/A", 0);
            }

            try
            {
                machineLogThreadCount = InitializePerfCounterInstance(ServiceEventSource, "Total event logs");
                machineLogActiveThreadCount = InitializePerfCounterInstance(ServiceEventSource, "Active event logs");
                eventEntriesPerSecond = InitializePerfCounterInstance(ServiceEventSource, "Event entries/Sec");
                errorsPerSecond = InitializePerfCounterInstance(ServiceEventSource, "Errors/Sec");
                dbAccessPerSecond = InitializePerfCounterInstance(ServiceEventSource, "Database accesses/Sec");
                dbInsertsPerSecond = InitializePerfCounterInstance(ServiceEventSource, "Database inserts/Sec");
                dbDuplicateInsertsPerSecond = InitializePerfCounterInstance(ServiceEventSource, "Database duplicate inserts/Sec");
                dbAccessDurationMilliseconds = InitializePerfCounterInstance(perfCategoryThreads, executionDurationTimeCounterName, "Database access time");
            }
            catch (Exception ex)
            {
                LogError(ex.ToString(), EventIDCodes.SystemEvents, null, "Creating performance counters", System.Net.Dns.GetHostName(), "N/A", 0);
            }
        }

        private static PerformanceCounter InitializePerfCounterInstance(
            string categoryName, string counterName)
        {
            PerformanceCounter counter = new PerformanceCounter(categoryName, counterName, false);
            counter.BeginInit();
            counter.RawValue = 0;
            counter.EndInit();
            return counter;
        }
        private static PerformanceCounter InitializePerfCounterInstance(
            string categoryName, string counterName, string instanceName)
        {
            PerformanceCounter counter = new PerformanceCounter(categoryName, counterName, instanceName, false);
            counter.BeginInit();
            counter.RawValue = 0;
            counter.EndInit();
            return counter;
        }
        private static void ClosePerformanceCounters()
        {
            try
            {
                while (machineLogThreadCount.RawValue > 0)
                    Thread.Sleep(1000);

                machineLogThreadCount.Close();
                machineLogThreadCount.Dispose();
            }
            catch (Exception ex)
            {
                LogError(ex.ToString(), EventIDCodes.SystemEvents, null, "Closing performance counter", System.Net.Dns.GetHostName(), "N/A", 0);
            }
        }
        private static void IncrementCounter(PerformanceCounter counter)
        {
            try
            {
                if (counter == null)
                {
                    LogError("Performance counter not set up or installed! Please verify installation!", EventIDCodes.SystemEvents, null, "Increment performance counter", System.Net.Dns.GetHostName(), "N/A", 0);
                }
                else
                {
                    counter.Increment();
                }
            }
            catch (Exception ex)
            {
                LogError(ex.ToString(), EventIDCodes.SystemEvents, null, "Increment performance counter", System.Net.Dns.GetHostName(), "N/A", 0);
            }
        }
        private static void IncrementCounter(PerformanceCounter counter, long incrementBy)
        {
            try
            {
                if (counter == null)
                {
                    LogError("Performance counter not set up or installed! Please verify installation!", EventIDCodes.SystemEvents, null, "Increment performance counter", System.Net.Dns.GetHostName(), "N/A", 0);
                }
                else
                {
                    counter.IncrementBy(incrementBy);
                }
            }
            catch (Exception ex)
            {
                LogError(ex.ToString(), EventIDCodes.SystemEvents, null, "Increment performance counter", System.Net.Dns.GetHostName(), "N/A", 0);
            }
        }
        private static void DecrementCounter(PerformanceCounter counter)
        {
            try
            {
                if (counter == null)
                {
                    LogError("Performance counter not set up or installed! Please verify installation!", EventIDCodes.SystemEvents, null, "Increment performance counter", System.Net.Dns.GetHostName(), "N/A", 0);
                }
                else
                {
                    counter.Decrement();
                }
            }
            catch (Exception ex)
            {
                LogError(ex.ToString(), EventIDCodes.SystemEvents, null, "Decrement performance counter", System.Net.Dns.GetHostName(), "N/A", 0);
            }
        }
        private static void SetThreadCounterActiveState(PerformanceCounter counter, bool isActive)
        {
            try
            {
                if (counter == null)
                {
                    LogError("Performance counter not set up or installed! Please verify installation!", EventIDCodes.SystemEvents, null, "Increment performance counter", System.Net.Dns.GetHostName(), "N/A", 0);
                }
                else
                {
                    if (isActive)
                    {
                        counter.BeginInit();
                        counter.RawValue = 10;
                        counter.EndInit();
                        //Thread.Sleep(1000); //give perf mon readers to catch the value
                    }
                    else
                    {
                        counter.BeginInit();
                        counter.RawValue = 0;
                        counter.EndInit();
                    }
                }
            }
            catch (Exception ex)
            {
                LogError(ex.ToString(), EventIDCodes.SystemEvents, null, "SetThreadCounterActiveState", System.Net.Dns.GetHostName(), "N/A", 0);
            }
        }
        #endregion

        #region Null handling
        private static string N(object anyVal)
        {
            if (anyVal == null)
                return "";
            else
                return anyVal.ToString();
        } 
        #endregion
    }
}
